In [1]:
%pylab
%load_ext autoreload
%autoreload 2
from datetime import datetime, timedelta
import os
import sys
import numpy as np
import pandas as pd
from scipy import sparse
import sklearn as sl
import theanets as tn
from sklearn.cross_validation import StratifiedKFold, cross_val_score
from sklearn.feature_selection import RFECV
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier
from sklearn.metrics import roc_auc_score, precision_recall_fscore_support
from cPickle import Pickler
Using matplotlib backend: Qt4Agg
Populating the interactive namespace from numpy and matplotlib
Couldn't import dot_parser, loading of dot files will not be possible.
Using gpu device 0: GeForce GTX 980
In [2]:
if os.name == 'nt':
TRAIN_PATH = r'D:\train.csv'
PTRAIN_PATH = r'D:\springleaf\train_preprocessed_all.csv'
TEST_PATH = r'D:\test.csv'
GOOGNEWS_PATH = r'D:\GoogleNews-vectors-negative300.bin.gz'
VOCAB_PATH = r'D:\big.txt'
else:
TRAIN_PATH = r'/media/mtambos/speedy/springleaf/train.csv'
PTRAIN_PATH = r'/media/mtambos/speedy/springleaf/train_preprocessed_all.csv'
TEST_PATH = r'/media/mtambos/speedy/springleaf/test.csv'
GOOGNEWS_PATH = r'/media/mtambos/speedy/GoogleNews-vectors-negative300.bin.gz'
VOCAB_PATH = r'/media/mtambos/speedy/big.txt'
df = pd.read_csv(PTRAIN_PATH, dtype=np.float32)
In [3]:
print "Calculating medians"
df_median = df.median(skipna=True)
print "Filling NAs"
df = df.fillna(value=df_median)
Calculating medians
Filling NAs
In [4]:
X = df.loc[:, df.columns != 'target']
y = df['target']
In [5]:
skf = StratifiedKFold(y, n_folds=5)
In [24]:
rf_classifier = RandomForestClassifier(n_estimators=100, n_jobs=-1, verbose=1, class_weight='auto')
rf_scores = cross_val_score(estimator=rf_classifier, X=X, y=y, scoring='roc_auc', cv=skf, verbose=2)
rf_scores
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.0s remaining: 4.9min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 39.1s finished
[Parallel(n_jobs=8)]: Done 1 out of 47 | elapsed: 0.0s remaining: 1.1s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
[CV] no parameters to be set .........................................
[CV] ................................ no parameters to be set - 40.0s
[Parallel(n_jobs=1)]: Done 1 jobs | elapsed: 40.0s
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.0s remaining: 4.9min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 38.1s finished
[Parallel(n_jobs=8)]: Done 1 out of 100 | elapsed: 0.0s remaining: 2.1s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
[CV] no parameters to be set .........................................
[CV] ................................ no parameters to be set - 39.1s
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 2.9s remaining: 4.8min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 37.9s finished
[Parallel(n_jobs=8)]: Done 1 out of 37 | elapsed: 0.0s remaining: 0.8s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.3s finished
[CV] no parameters to be set .........................................
[CV] ................................ no parameters to be set - 38.8s
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.0s remaining: 5.0min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 36.9s finished
[Parallel(n_jobs=8)]: Done 1 out of 52 | elapsed: 0.0s remaining: 1.0s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
[CV] no parameters to be set .........................................
[CV] ................................ no parameters to be set - 37.8s
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 2.9s remaining: 4.8min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 37.4s finished
[Parallel(n_jobs=8)]: Done 1 out of 85 | elapsed: 0.0s remaining: 2.0s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
[CV] no parameters to be set .........................................
[CV] ................................ no parameters to be set - 38.3s
[Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 3.2min finished
Out[24]:
array([ 0.75984939, 0.76191922, 0.75785135, 0.7627222 , 0.75827303])
In [11]:
classifier.fit(X=X, y=y)
hundred_most_important_features = np.argpartition(classifier.feature_importances_, range(50))[:50]
df.columns[hundred_most_important_features]
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 5.1s remaining: 8.5min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 1.1min finished
Out[11]:
Index([u'VAR_0098', u'VAR_0106', u'VAR_0138', u'VAR_0191', u'VAR_0193',
u'VAR_0214', u'VAR_0396', u'VAR_0445', u'VAR_0526', u'VAR_0529',
u'VAR_0192', u'VAR_0107', u'VAR_0395', u'VAR_0397', u'VAR_0114',
u'VAR_0194', u'VAR_0130', u'VAR_0398', u'VAR_0459', u'VAR_0399',
u'VAR_0182', u'VAR_0463', u'VAR_0180', u'VAR_0131', u'VAR_0195',
u'VAR_0392', u'VAR_0181', u'VAR_0411', u'VAR_0139', u'VAR_0393',
u'VAR_0115', u'VAR_0449', u'VAR_0277', u'VAR_0108', u'VAR_0507',
u'VAR_0099', u'VAR_0603', u'VAR_0468', u'VAR_0437', u'VAR_0275',
u'VAR_0428', u'VAR_0386', u'VAR_0460', u'VAR_0371', u'VAR_1012',
u'VAR_0156_year', u'VAR_1849', u'VAR_0476', u'VAR_0521', u'VAR_0229'],
dtype='object')
In [14]:
hundred_most_important_features = sorted(classifier.feature_importances_, reverse=True)[:50]
hundred_most_important_features
Out[14]:
[0.0028886732507686784,
0.0028240274642443606,
0.0027145478135424438,
0.002649123171116595,
0.0026337071206189132,
0.0025774113335795806,
0.0025577297698921426,
0.0024156090101467576,
0.0023696137063085108,
0.0023295319171059235,
0.0022627918326271926,
0.0022563778887519058,
0.0022293222963358796,
0.002217220855337247,
0.002207549625493133,
0.0021855396659237925,
0.0021794922217861744,
0.0021612128635470776,
0.0021464625567381075,
0.0020887284358406533,
0.0020847885614810077,
0.0020715077656085924,
0.0020212881690338147,
0.0019848917080766697,
0.0019481709547825412,
0.0019168199587121636,
0.0019082153391379838,
0.0018996953398093921,
0.0018862264807112249,
0.0018613543956342157,
0.0018567161214247685,
0.001840180358874736,
0.0018355054539040527,
0.0018309183724551155,
0.0018186855540627753,
0.001812398493039133,
0.0018111956642958996,
0.0018089679575626412,
0.0018078433546637178,
0.0018075085449324763,
0.001804660508160783,
0.0017879927547157614,
0.0017817126624673359,
0.0017520622281477821,
0.0017424798374334751,
0.0017367168562071355,
0.0017351106598247736,
0.0017306116877069367,
0.0017259164970448659,
0.0017151883570456045]
In [22]:
et_classifier = ExtraTreesClassifier(n_estimators=100, n_jobs=-1, verbose=2, class_weight='auto')
et_scores = cross_val_score(estimator=et_classifier, X=X, y=y, scoring='roc_auc', cv=skf, verbose=2, n_jobs=-1)
et_scores
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 4.4s remaining: 7.2min
[Parallel(n_jobs=-1)]: Done 51 out of 100 | elapsed: 30.9s remaining: 29.7s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 55.8s finished
[Parallel(n_jobs=8)]: Done 1 out of 100 | elapsed: 0.0s remaining: 3.3s
[Parallel(n_jobs=8)]: Done 51 out of 100 | elapsed: 0.2s remaining: 0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.5s finished
[CV] no parameters to be set .........................................
building tree 1 of 100
building tree 2 of 100
building tree 3 of 100
building tree 4 of 100
building tree 5 of 100
building tree 6 of 100
building tree 7 of 100
building tree 8 of 100
building tree 9 of 100
building tree 10 of 100
building tree 11 of 100
building tree 12 of 100
building tree 13 of 100
building tree 14 of 100
building tree 15 of 100
building tree 16 of 100
building tree 17 of 100
building tree 18 of 100
building tree 19 of 100
building tree 20 of 100
building tree 21 of 100
building tree 22 of 100
building tree 23 of 100
building tree 24 of 100
building tree 25 of 100
building tree 26 of 100
building tree 27 of 100
building tree 28 of 100
building tree 29 of 100
building tree 30 of 100
building tree 31 of 100
building tree 32 of 100
building tree 33 of 100
building tree 34 of 100
building tree 35 of 100
building tree 36 of 100
building tree 37 of 100
building tree 38 of 100
building tree 39 of 100
building tree 40 of 100
building tree 41 of 100
building tree 42 of 100
building tree 43 of 100
building tree 44 of 100
building tree 45 of 100
building tree 46 of 100
building tree 47 of 100
building tree 48 of 100
building tree 49 of 100
building tree 50 of 100
building tree 51 of 100
building tree 52 of 100
building tree 53 of 100
building tree 54 of 100
building tree 55 of 100
building tree 56 of 100
building tree 57 of 100
building tree 58 of 100
building tree 59 of 100
building tree 60 of 100
building tree 61 of 100
building tree 62 of 100
building tree 63 of 100
building tree 64 of 100
building tree 65 of 100
building tree 66 of 100
building tree 67 of 100
building tree 68 of 100
building tree 69 of 100
building tree 70 of 100
building tree 71 of 100
building tree 72 of 100
building tree 73 of 100
building tree 74 of 100
building tree 75 of 100
building tree 76 of 100
building tree 77 of 100
building tree 78 of 100
building tree 79 of 100
building tree 80 of 100
building tree 81 of 100
building tree 82 of 100
building tree 83 of 100
building tree 84 of 100
building tree 85 of 100
building tree 86 of 100
building tree 87 of 100
building tree 88 of 100
building tree 89 of 100
building tree 90 of 100
building tree 91 of 100
building tree 92 of 100
building tree 93 of 100
building tree 94 of 100
building tree 95 of 100
building tree 96 of 100
building tree 97 of 100
building tree 98 of 100
building tree 99 of 100
building tree 100 of 100
[CV] ................................ no parameters to be set - 56.8s
[Parallel(n_jobs=1)]: Done 1 jobs | elapsed: 56.8s
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 5.3s remaining: 8.7min
[Parallel(n_jobs=-1)]: Done 51 out of 100 | elapsed: 30.7s remaining: 29.5s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 56.0s finished
[Parallel(n_jobs=8)]: Done 1 out of 86 | elapsed: 0.0s remaining: 2.7s
[Parallel(n_jobs=8)]: Done 51 out of 100 | elapsed: 0.2s remaining: 0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.5s finished
[CV] no parameters to be set .........................................
building tree 1 of 100
building tree 2 of 100
building tree 3 of 100
building tree 5 of 100
building tree 4 of 100
building tree 6 of 100
building tree 7 of 100
building tree 8 of 100
building tree 9 of 100
building tree 10 of 100
building tree 11 of 100
building tree 12 of 100
building tree 13 of 100
building tree 14 of 100
building tree 15 of 100
building tree 16 of 100
building tree 17 of 100
building tree 18 of 100
building tree 19 of 100
building tree 20 of 100
building tree 21 of 100
building tree 22 of 100
building tree 23 of 100
building tree 24 of 100
building tree 25 of 100
building tree 26 of 100
building tree 27 of 100
building tree 28 of 100
building tree 29 of 100
building tree 30 of 100
building tree 31 of 100
building tree 32 of 100
building tree 33 of 100
building tree 34 of 100
building tree 35 of 100
building tree 36 of 100
building tree 37 of 100
building tree 38 of 100
building tree 39 of 100
building tree 40 of 100
building tree 41 of 100
building tree 42 of 100
building tree 43 of 100
building tree 44 of 100
building tree 45 of 100
building tree 46 of 100
building tree 47 of 100
building tree 48 of 100
building tree 49 of 100
building tree 50 of 100
building tree 51 of 100
building tree 52 of 100
building tree 53 of 100
building tree 54 of 100
building tree 55 of 100
building tree 56 of 100
building tree 57 of 100
building tree 58 of 100
building tree 59 of 100
building tree 60 of 100
building tree 61 of 100
building tree 62 of 100
building tree 63 of 100
building tree 64 of 100
building tree 65 of 100
building tree 66 of 100
building tree 67 of 100
building tree 68 of 100
building tree 69 of 100
building tree 70 of 100
building tree 71 of 100
building tree 72 of 100
building tree 73 of 100
building tree 74 of 100
building tree 75 of 100
building tree 76 of 100
building tree 77 of 100
building tree 78 of 100
building tree 79 of 100
building tree 80 of 100
building tree 81 of 100
building tree 82 of 100
building tree 83 of 100
building tree 84 of 100
building tree 85 of 100
building tree 86 of 100
building tree 87 of 100
building tree 88 of 100
building tree 89 of 100
building tree 90 of 100
building tree 91 of 100
building tree 92 of 100
building tree 93 of 100
building tree 94 of 100
building tree 95 of 100
building tree 96 of 100
building tree 97 of 100
building tree 98 of 100
building tree 99 of 100
building tree 100 of 100
[CV] ................................ no parameters to be set - 57.0s
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 4.4s remaining: 7.2min
[Parallel(n_jobs=-1)]: Done 51 out of 100 | elapsed: 30.3s remaining: 29.1s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 54.6s finished
[Parallel(n_jobs=8)]: Done 1 out of 100 | elapsed: 0.0s remaining: 3.3s
[Parallel(n_jobs=8)]: Done 51 out of 100 | elapsed: 0.3s remaining: 0.3s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.5s finished
[CV] no parameters to be set .........................................
building tree 1 of 100
building tree 2 of 100
building tree 3 of 100
building tree 4 of 100
building tree 5 of 100
building tree 6 of 100
building tree 7 of 100
building tree 8 of 100
building tree 9 of 100
building tree 10 of 100
building tree 11 of 100
building tree 12 of 100
building tree 13 of 100
building tree 14 of 100
building tree 15 of 100
building tree 16 of 100
building tree 17 of 100
building tree 18 of 100
building tree 19 of 100
building tree 20 of 100
building tree 21 of 100
building tree 22 of 100
building tree 23 of 100
building tree 24 of 100
building tree 25 of 100
building tree 26 of 100
building tree 27 of 100
building tree 28 of 100
building tree 29 of 100
building tree 30 of 100
building tree 31 of 100
building tree 32 of 100
building tree 33 of 100
building tree 34 of 100
building tree 35 of 100
building tree 36 of 100
building tree 37 of 100
building tree 38 of 100
building tree 39 of 100
building tree 40 of 100
building tree 41 of 100
building tree 42 of 100
building tree 43 of 100
building tree 44 of 100
building tree 45 of 100
building tree 46 of 100
building tree 47 of 100
building tree 48 of 100
building tree 49 of 100
building tree 50 of 100
building tree 51 of 100
building tree 52 of 100
building tree 53 of 100
building tree 54 of 100
building tree 55 of 100
building tree 56 of 100
building tree 57 of 100
building tree 58 of 100
building tree 59 of 100
building tree 60 of 100
building tree 61 of 100
building tree 62 of 100
building tree 63 of 100
building tree 64 of 100
building tree 65 of 100
building tree 66 of 100
building tree 67 of 100
building tree 68 of 100
building tree 69 of 100
building tree 70 of 100
building tree 71 of 100
building tree 72 of 100
building tree 73 of 100
building tree 74 of 100
building tree 75 of 100
building tree 76 of 100
building tree 77 of 100
building tree 78 of 100
building tree 79 of 100
building tree 80 of 100
building tree 81 of 100
building tree 82 of 100
building tree 83 of 100
building tree 84 of 100
building tree 85 of 100
building tree 86 of 100
building tree 87 of 100
building tree 88 of 100
building tree 89 of 100
building tree 90 of 100
building tree 91 of 100
building tree 92 of 100
building tree 93 of 100
building tree 94 of 100
building tree 95 of 100
building tree 96 of 100
building tree 97 of 100
building tree 98 of 100
building tree 99 of 100
building tree 100 of 100
[CV] ................................ no parameters to be set - 55.6s
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 4.3s remaining: 7.1min
[Parallel(n_jobs=-1)]: Done 51 out of 100 | elapsed: 29.3s remaining: 28.1s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 55.0s finished
[Parallel(n_jobs=8)]: Done 1 out of 57 | elapsed: 0.0s remaining: 1.9s
[Parallel(n_jobs=8)]: Done 51 out of 100 | elapsed: 0.3s remaining: 0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.5s finished
[CV] no parameters to be set .........................................
building tree 1 of 100
building tree 2 of 100
building tree 3 of 100
building tree 4 of 100
building tree 5 of 100
building tree 6 of 100
building tree 7 of 100
building tree 8 of 100
building tree 9 of 100
building tree 10 of 100
building tree 11 of 100
building tree 12 of 100
building tree 13 of 100
building tree 14 of 100
building tree 15 of 100
building tree 16 of 100
building tree 17 of 100
building tree 18 of 100
building tree 19 of 100
building tree 20 of 100
building tree 21 of 100
building tree 22 of 100
building tree 23 of 100
building tree 24 of 100
building tree 25 of 100
building tree 26 of 100
building tree 27 of 100
building tree 28 of 100
building tree 29 of 100
building tree 30 of 100
building tree 31 of 100
building tree 32 of 100
building tree 33 of 100
building tree 34 of 100
building tree 35 of 100
building tree 36 of 100
building tree 37 of 100
building tree 38 of 100
building tree 39 of 100
building tree 40 of 100
building tree 41 of 100
building tree 42 of 100
building tree 43 of 100
building tree 44 of 100
building tree 45 of 100
building tree 46 of 100
building tree 47 of 100
building tree 48 of 100
building tree 49 of 100
building tree 50 of 100
building tree 51 of 100
building tree 52 of 100
building tree 53 of 100
building tree 54 of 100
building tree 55 of 100
building tree 56 of 100
building tree 57 of 100
building tree 58 of 100
building tree 59 of 100
building tree 60 of 100
building tree 61 of 100
building tree 62 of 100
building tree 63 of 100
building tree 64 of 100
building tree 65 of 100
building tree 66 of 100
building tree 67 of 100
building tree 68 of 100
building tree 69 of 100
building tree 70 of 100
building tree 71 of 100
building tree 72 of 100
building tree 73 of 100
building tree 74 of 100
building tree 75 of 100
building tree 76 of 100
building tree 77 of 100
building tree 78 of 100
building tree 79 of 100
building tree 80 of 100
building tree 81 of 100
building tree 82 of 100
building tree 83 of 100
building tree 84 of 100
building tree 85 of 100
building tree 86 of 100
building tree 87 of 100
building tree 88 of 100
building tree 89 of 100
building tree 90 of 100
building tree 91 of 100
building tree 92 of 100
building tree 93 of 100
building tree 94 of 100
building tree 95 of 100
building tree 96 of 100
building tree 97 of 100
building tree 98 of 100
building tree 99 of 100
building tree 100 of 100
[CV] ................................ no parameters to be set - 56.1s
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 4.3s remaining: 7.1min
[Parallel(n_jobs=-1)]: Done 51 out of 100 | elapsed: 29.3s remaining: 28.2s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 53.7s finished
[Parallel(n_jobs=8)]: Done 1 out of 33 | elapsed: 0.0s remaining: 1.1s
[Parallel(n_jobs=8)]: Done 51 out of 100 | elapsed: 0.3s remaining: 0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.5s finished
[CV] no parameters to be set .........................................
building tree 1 of 100building tree 2 of 100building tree 3 of 100building tree 4 of 100
building tree 6 of 100building tree 7 of 100building tree 8 of 100
building tree 5 of 100
building tree 9 of 100
building tree 10 of 100
building tree 11 of 100
building tree 12 of 100
building tree 13 of 100
building tree 14 of 100
building tree 15 of 100
building tree 16 of 100
building tree 17 of 100
building tree 18 of 100
building tree 19 of 100
building tree 20 of 100
building tree 21 of 100
building tree 22 of 100
building tree 23 of 100
building tree 24 of 100
building tree 25 of 100
building tree 26 of 100
building tree 27 of 100
building tree 28 of 100
building tree 29 of 100
building tree 30 of 100
building tree 31 of 100
building tree 32 of 100
building tree 33 of 100
building tree 34 of 100
building tree 35 of 100
building tree 36 of 100
building tree 37 of 100
building tree 38 of 100
building tree 39 of 100
building tree 40 of 100
building tree 41 of 100
building tree 42 of 100
building tree 43 of 100
building tree 44 of 100
building tree 45 of 100
building tree 46 of 100
building tree 47 of 100
building tree 48 of 100
building tree 49 of 100
building tree 50 of 100
building tree 51 of 100
building tree 52 of 100
building tree 53 of 100
building tree 54 of 100
building tree 55 of 100
building tree 56 of 100
building tree 57 of 100
building tree 58 of 100
building tree 59 of 100
building tree 60 of 100
building tree 61 of 100
building tree 62 of 100
building tree 63 of 100
building tree 64 of 100
building tree 65 of 100
building tree 66 of 100
building tree 67 of 100
building tree 68 of 100
building tree 69 of 100
building tree 70 of 100
building tree 71 of 100
building tree 72 of 100
building tree 73 of 100
building tree 74 of 100
building tree 75 of 100
building tree 76 of 100
building tree 77 of 100
building tree 78 of 100
building tree 79 of 100
building tree 80 of 100
building tree 81 of 100
building tree 82 of 100
building tree 83 of 100
building tree 84 of 100
building tree 85 of 100
building tree 86 of 100
building tree 87 of 100
building tree 88 of 100
building tree 89 of 100
building tree 90 of 100
building tree 91 of 100
building tree 92 of 100
building tree 93 of 100
building tree 94 of 100
building tree 95 of 100
building tree 96 of 100
building tree 97 of 100
building tree 98 of 100
building tree 99 of 100
building tree 100 of 100
[CV] ................................ no parameters to be set - 54.7s
[Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 4.7min finished
Out[22]:
array([ 0.75727822, 0.75785649, 0.75725815, 0.76027825, 0.75470654])
In [28]:
ab_classifier = AdaBoostClassifier(n_estimators=100)
ab_scores = cross_val_score(estimator=ab_classifier, X=X, y=y, scoring='roc_auc', cv=skf, verbose=2, n_jobs=-1)
ab_scores
[Parallel(n_jobs=-1)]: Done 1 jobs | elapsed: 13.3min
[Parallel(n_jobs=-1)]: Done 3 out of 5 | elapsed: 13.6min remaining: 9.1min
[Parallel(n_jobs=-1)]: Done 5 out of 5 | elapsed: 13.6min finished
[CV] no parameters to be set .........................................
[CV] no parameters to be set .........................................
[CV] no parameters to be set .........................................
[CV] no parameters to be set .........................................
[CV] no parameters to be set .........................................
[CV] ................................ no parameters to be set -13.3min[CV] ................................ no parameters to be set -13.3min[CV] ................................ no parameters to be set -13.6min[CV] ................................ no parameters to be set -13.3min[CV] ................................ no parameters to be set -13.3min
Out[28]:
array([ 0.75965457, 0.76372955, 0.76249127, 0.76381007, 0.76312906])
In [ ]:
gb_classifier = GradientBoostingClassifier(n_estimators=100, verbose=1)
gb_scores = cross_val_score(estimator=gb_classifier, X=X, y=y, scoring='roc_auc', cv=skf,
verbose=2, n_jobs=4, pre_dispatch='n_jobs')
gb_scores
In [19]:
class RandomForestClassifierWithCoef(RandomForestClassifier):
"""
Taken from http://stackoverflow.com/a/24656474/1632574
"""
def fit(self, *args, **kwargs):
super(RandomForestClassifierWithCoef, self).fit(*args, **kwargs)
self.coef_ = self.feature_importances_
In [20]:
classifier2 = RandomForestClassifierWithCoef(n_estimators=100, n_jobs=-1, verbose=1)
rfecv = RFECV(estimator=classifier2, step=0.1, cv=skf, scoring='roc_auc', verbose=2)
rfecv.fit(X=X, y=y)
rfecv.n_features_
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 4.5s remaining: 7.4min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 53.6s finished
Fitting estimator with 1905 features.
Fitting estimator with 1715 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 4.4s remaining: 7.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 51.4s finished
Fitting estimator with 1525 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.9s remaining: 6.4min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 49.6s finished
Fitting estimator with 1335 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.8s remaining: 6.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 47.2s finished
Fitting estimator with 1145 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.8s remaining: 6.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 44.1s finished
Fitting estimator with 955 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.4s remaining: 5.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 41.6s finished
Fitting estimator with 765 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.2s remaining: 5.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 39.4s finished
Fitting estimator with 575 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 2.7s remaining: 4.4min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 34.9s finished
Fitting estimator with 385 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 2.5s remaining: 4.2min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 29.5s finished
Fitting estimator with 195 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 1.5s remaining: 2.5min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 19.5s finished
Fitting estimator with 5 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 0.2s remaining: 21.5s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 2.5s finished
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 0.2s remaining: 16.1s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 2.1s finished
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 0.2s remaining: 16.9s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 2.0s finished
[Parallel(n_jobs=8)]: Done 1 out of 9 | elapsed: 0.0s remaining: 0.0s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.1s finished
Finished fold with 1 / 12 feature ranks, score=0.563906
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 0.2s remaining: 19.5s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 2.5s finished
[Parallel(n_jobs=8)]: Done 1 out of 12 | elapsed: 0.0s remaining: 0.1s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.1s finished
Finished fold with 2 / 12 feature ranks, score=0.632698
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 1.6s remaining: 2.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 19.6s finished
[Parallel(n_jobs=8)]: Done 1 out of 100 | elapsed: 0.0s remaining: 1.9s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.2s finished
Finished fold with 3 / 12 feature ranks, score=0.756228
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 2.3s remaining: 3.8min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 29.2s finished
[Parallel(n_jobs=8)]: Done 1 out of 100 | elapsed: 0.0s remaining: 2.7s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.3s finished
Finished fold with 4 / 12 feature ranks, score=0.760178
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 2.8s remaining: 4.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 34.5s finished
[Parallel(n_jobs=8)]: Done 1 out of 98 | elapsed: 0.0s remaining: 1.7s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.3s finished
Finished fold with 5 / 12 feature ranks, score=0.760337
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.0s remaining: 5.0min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 38.4s finished
[Parallel(n_jobs=8)]: Done 1 out of 29 | elapsed: 0.0s remaining: 0.5s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
Finished fold with 6 / 12 feature ranks, score=0.760583
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.1s remaining: 5.2min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 39.2s finished
[Parallel(n_jobs=8)]: Done 1 out of 100 | elapsed: 0.0s remaining: 2.3s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.3s finished
Finished fold with 7 / 12 feature ranks, score=0.760868
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.5s remaining: 5.7min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 41.3s finished
[Parallel(n_jobs=8)]: Done 1 out of 69 | elapsed: 0.0s remaining: 1.6s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
Finished fold with 8 / 12 feature ranks, score=0.758182
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.7s remaining: 6.1min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 43.9s finished
[Parallel(n_jobs=8)]: Done 1 out of 78 | elapsed: 0.0s remaining: 1.6s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
Finished fold with 9 / 12 feature ranks, score=0.760035
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.6s remaining: 6.0min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 46.1s finished
[Parallel(n_jobs=8)]: Done 1 out of 23 | elapsed: 0.0s remaining: 0.5s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.5s finished
Finished fold with 10 / 12 feature ranks, score=0.757785
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 4.1s remaining: 6.7min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 47.3s finished
[Parallel(n_jobs=8)]: Done 1 out of 100 | elapsed: 0.0s remaining: 3.6s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.5s finished
Finished fold with 11 / 12 feature ranks, score=0.759269
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.9s remaining: 6.4min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 48.3s finished
[Parallel(n_jobs=8)]: Done 1 out of 100 | elapsed: 0.0s remaining: 2.7s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
Finished fold with 12 / 12 feature ranks, score=0.759254
Fitting estimator with 1905 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.9s remaining: 6.4min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 47.0s finished
Fitting estimator with 1715 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.8s remaining: 6.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 46.6s finished
Fitting estimator with 1525 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 4.0s remaining: 6.5min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 47.5s finished
Fitting estimator with 1335 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 4.0s remaining: 6.7min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 46.0s finished
Fitting estimator with 1145 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.8s remaining: 6.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 43.8s finished
Fitting estimator with 955 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.8s remaining: 6.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 43.8s finished
Fitting estimator with 765 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.1s remaining: 5.1min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 40.9s finished
Fitting estimator with 575 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 2.8s remaining: 4.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 35.0s finished
Fitting estimator with 385 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 2.4s remaining: 3.9min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 30.1s finished
Fitting estimator with 195 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 1.7s remaining: 2.8min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 20.7s finished
Fitting estimator with 5 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 0.2s remaining: 17.4s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 2.2s finished
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 0.2s remaining: 18.0s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 2.1s finished
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 0.2s remaining: 15.2s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 2.0s finished
[Parallel(n_jobs=8)]: Done 1 out of 39 | elapsed: 0.0s remaining: 0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.1s finished
Finished fold with 1 / 12 feature ranks, score=0.562396
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 0.2s remaining: 16.8s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 2.3s finished
[Parallel(n_jobs=8)]: Done 1 out of 15 | elapsed: 0.0s remaining: 0.1s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.2s finished
Finished fold with 2 / 12 feature ranks, score=0.639067
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 1.6s remaining: 2.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 20.2s finished
[Parallel(n_jobs=8)]: Done 1 out of 29 | elapsed: 0.0s remaining: 0.5s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.2s finished
Finished fold with 3 / 12 feature ranks, score=0.755461
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 2.4s remaining: 3.9min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 29.9s finished
[Parallel(n_jobs=8)]: Done 1 out of 18 | elapsed: 0.0s remaining: 0.3s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.3s finished
Finished fold with 4 / 12 feature ranks, score=0.758866
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 2.7s remaining: 4.5min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 37.7s finished
[Parallel(n_jobs=8)]: Done 1 out of 100 | elapsed: 0.0s remaining: 2.3s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.5s finished
Finished fold with 5 / 12 feature ranks, score=0.760468
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.4s remaining: 5.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 40.6s finished
[Parallel(n_jobs=8)]: Done 1 out of 95 | elapsed: 0.0s remaining: 1.8s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
Finished fold with 6 / 12 feature ranks, score=0.760325
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.2s remaining: 5.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 43.9s finished
[Parallel(n_jobs=8)]: Done 1 out of 35 | elapsed: 0.0s remaining: 0.9s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
Finished fold with 7 / 12 feature ranks, score=0.761522
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.3s remaining: 5.5min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 42.9s finished
[Parallel(n_jobs=8)]: Done 1 out of 100 | elapsed: 0.0s remaining: 3.1s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.5s finished
Finished fold with 8 / 12 feature ranks, score=0.761705
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.9s remaining: 6.4min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 45.4s finished
[Parallel(n_jobs=8)]: Done 1 out of 100 | elapsed: 0.0s remaining: 4.9s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
Finished fold with 9 / 12 feature ranks, score=0.762252
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.8s remaining: 6.2min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 46.7s finished
[Parallel(n_jobs=8)]: Done 1 out of 47 | elapsed: 0.0s remaining: 1.3s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.5s finished
Finished fold with 10 / 12 feature ranks, score=0.761487
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 4.0s remaining: 6.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 47.8s finished
[Parallel(n_jobs=8)]: Done 1 out of 9 | elapsed: 0.0s remaining: 0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
Finished fold with 11 / 12 feature ranks, score=0.762477
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.7s remaining: 6.0min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 49.9s finished
[Parallel(n_jobs=8)]: Done 1 out of 8 | elapsed: 0.0s remaining: 0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed: 0.4s finished
Finished fold with 12 / 12 feature ranks, score=0.761509
Fitting estimator with 1905 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 4.2s remaining: 6.9min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 48.4s finished
Fitting estimator with 1715 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 3.8s remaining: 6.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 47.9s finished
Fitting estimator with 1525 features.
[Parallel(n_jobs=-1)]: Done 1 out of 100 | elapsed: 4.2s remaining: 7.0min
---------------------------------------------------------------------------
KeyboardInterrupt Traceback (most recent call last)
<ipython-input-20-9fd5344f1c5e> in <module>()
1 classifier2 = RandomForestClassifierWithCoef(n_estimators=100, n_jobs=-1, verbose=1)
2 rfecv = RFECV(estimator=classifier2, step=0.1, cv=skf, scoring='roc_auc', verbose=2)
----> 3 rfecv.fit(X=X, y=y)
4 rfecv.n_features_
/home/mtambos/anaconda/lib/python2.7/site-packages/sklearn/feature_selection/rfe.pyc in fit(self, X, y)
374 # ranking_ contains the same set of values for all CV folds,
375 # but perhaps reordered
--> 376 ranking_ = rfe.fit(X_train, y_train).ranking_
377 # Score each subset of features
378 for k in range(0, np.max(ranking_)):
/home/mtambos/anaconda/lib/python2.7/site-packages/sklearn/feature_selection/rfe.pyc in fit(self, X, y)
159 print("Fitting estimator with %d features." % np.sum(support_))
160
--> 161 estimator.fit(X[:, features], y)
162
163 if estimator.coef_.ndim > 1:
<ipython-input-19-d94be2a1e7a5> in fit(self, *args, **kwargs)
1 class RandomForestClassifierWithCoef(RandomForestClassifier):
2 def fit(self, *args, **kwargs):
----> 3 super(RandomForestClassifierWithCoef, self).fit(*args, **kwargs)
4 self.coef_ = self.feature_importances_
/home/mtambos/anaconda/lib/python2.7/site-packages/sklearn/ensemble/forest.pyc in fit(self, X, y, sample_weight)
271 t, self, X, y, sample_weight, i, len(trees),
272 verbose=self.verbose, class_weight=self.class_weight)
--> 273 for i, t in enumerate(trees))
274
275 # Collect newly grown trees
/home/mtambos/anaconda/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc in __call__(self, iterable)
664 # consumption.
665 self._iterating = False
--> 666 self.retrieve()
667 # Make sure that we get a last message telling us we are done
668 elapsed_time = time.time() - self._start_time
/home/mtambos/anaconda/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc in retrieve(self)
516 self._lock.release()
517 try:
--> 518 self._output.append(job.get())
519 except tuple(self.exceptions) as exception:
520 try:
/home/mtambos/anaconda/lib/python2.7/multiprocessing/pool.pyc in get(self, timeout)
559
560 def get(self, timeout=None):
--> 561 self.wait(timeout)
562 if not self._ready:
563 raise TimeoutError
/home/mtambos/anaconda/lib/python2.7/multiprocessing/pool.pyc in wait(self, timeout)
554 try:
555 if not self._ready:
--> 556 self._cond.wait(timeout)
557 finally:
558 self._cond.release()
/home/mtambos/anaconda/lib/python2.7/threading.pyc in wait(self, timeout)
338 try: # restore state no matter what (e.g., KeyboardInterrupt)
339 if timeout is None:
--> 340 waiter.acquire()
341 if __debug__:
342 self._note("%s.wait(): got it", self)
KeyboardInterrupt:
Content source: mtambos/springleaf
Similar notebooks: